import requests

from lxml import etree


import threading

import time
import pymysql
from twisted.enterprise import adbapi

list_all = []


class Obtain:

    def __init__(self):

        self.headers = {
            'cache-control': 'no-cache',
            'cookie': 'session-id=130-2322939-7739406; session-id-time=2082787201l; i18n-prefs=USD; ubid-main=135-3888102-9406257; sp-cdn="L5Z9:TW"; session-token=L547CyHrI797TpcVGi9GZKHjNTXixkBViL+zw1hgOWD6X6DSP4yLBdMHJiGWKOaL3q1PvyfQ14RXG79syhJiuKVTlnwzw2rPqg1zy9G7Vwdp6j2CwZVNfIXjJeR60Y6fz+3odT3/90uVpQePMln+8T2UKmMNgfinKVx7G/gj2z2346/Rhgy71AdS4IrbLIA3; csm-hit=tb:5S3HN6WV2W7GKR3D6SP4+s-9EDEM9Q2YB1A1EBSPD2E|1629943795000&t:1629943795000&adb:adblk_no',
            'downlink': '8.75',
            'ect': '4g',
            'pragma': 'no-cache',
            'referer': 'https://www.amazon.com/Best-Sellers-Appstore-Android-Music-Audio/zgbs/mobile-apps/9408771011/ref=zg_bs_nav_mas_1_mas',
            'rtt': '250',
            'sec-ch-ua': '"Chromium";v="92", " Not A;Brand";v="99", "Google Chrome";v="92"',
            'sec-ch-ua-mobile': '?0',
            'sec-fetch-dest': 'document',
            'sec-fetch-mode': 'navigate',
            'sec-fetch-site': 'same-origin',
            'sec-fetch-user': '?1',
            'upgrade-insecure-requests': '1',
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.107 Safari/537.36',
        }
        # self.conn = pymysql.connect(host='192.168.2.205', port=3306, user='jinhuitao',
        #                        password='jinhuitao_123456', database='sunway_db', charset='utf8')
        # # 创建游标
        # self.cursor = self.conn.cursor()
        self.newtime = time.strftime("%Y-%m-%d", time.localtime())
        # self.newtime = '2021-10-23'

    def level_one(self, url):

        response = requests.get(url, headers=self.headers, verify=False, timeout=10).content.decode()
        # print(response)#第一层源代码
        # print('-'*10)
        l_HTML1 = etree.HTML(response)
        # print(l_HTML1)
        li_link = l_HTML1.xpath("//div[@id='zg_left_col2']/ul[@id='zg_browseRoot']/ul/li/a/@href")
        li_name = l_HTML1.xpath("//div[@id='zg_left_col2']/ul[@id='zg_browseRoot']/ul/li/a/text()")
        ioo = open('amazon_now.csv', 'w', encoding='utf-8')
        list_ts = []
        time01 = time.time()
        for i in range(len(li_name)):
            print('-' * 20)
            list_i = li_name[i]
            print(li_link[i], li_name[i])
            print('*' * 20)
            # semaphore = threading.Semaphore(30)  # 最大4个线程同时进行
            ts = threading.Thread(target=self.level_two, args=(li_link[i], list_i, ioo, li_name[i],))
            list_ts.append(ts)

            list_i = list_i.replace(f"->{li_name[i]}", "")
            print(22, list_i)

        for i_ts1 in list_ts:
            i_ts1.start()  # 启动线程

        for i_ts in list_ts:
            i_ts.join()  # 关闭线程
        time02 = time.time()
        print("时间:", time02 - time01)
        print(time.strftime("%H:%M:%S", time.localtime()))

    def level_two(self, url, list_i, ioo, name):
        try:
            response = requests.get(url, headers=self.headers, verify=False, timeout=10).content.decode()

            l_HTML2 = etree.HTML(response)
            # print(l_HTML2)
            print(url)
            print('*' * 20)
            li_link = l_HTML2.xpath("//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/li/a/@href")
            li_name = l_HTML2.xpath("//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/li/a/text()")

            if li_name != []:
                for i2 in range(len(li_link)):
                    print(li_link[i2], li_name[i2])
                    print('#2' * 20)
                    list_i = list_i + '->' + li_name[i2]
                    self.level_three(li_link[i2], list_i, ioo, li_name[i2])
                    list_i = list_i.replace(f"->{li_name[i2]}", "")
                    print(33, list_i)
            else:
                print(2, list_i)
                # ioo.write(f'{list_i}\n')
                # sql = 'insert into amazon_class(`list_i`,`newtime`) values("%s", "%s")'
                # self.cursor.execute(sql, (list_i, self.newtime))
                # self.conn.commit()
                list_all.append(list_i)

        except Exception as g:
            self.level_two(url, list_i, ioo, name)

    def level_three(self, url, list_i, ioo, name):
        try:
            response = requests.get(url, headers=self.headers, verify=False, timeout=10).content.decode()

            l_HTML3 = etree.HTML(response)

            li_link = l_HTML3.xpath("//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/li/a/@href")
            li_name = l_HTML3.xpath("//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/li/a/text()")

            if li_name != []:
                for i3 in range(len(li_link)):
                    print(li_link[i3], li_name[i3])
                    print('#3' * 20)
                    list_i = list_i + '->' + li_name[i3]
                    self.level_four(li_link[i3], list_i, ioo, li_name[i3])
                    list_i = list_i.replace(f"->{li_name[i3]}", "")
                    print(44, list_i)
            else:
                print(3, list_i)
                # ioo.write(f'{list_i}\n')
                # sql = 'insert into amazon_class(`list_i`,`newtime`) values("%s", "%s")'
                # self.cursor.execute(sql, (list_i, self.newtime))
                # self.conn.commit()
                # self.ioo.close()
                print(333)
                list_all.append(list_i)

        except Exception as g:
            self.level_three(url, list_i, ioo, name)

    def level_four(self, url, list_i, ioo, name):
        try:
            response = requests.get(url, headers=self.headers, verify=False, timeout=10).content.decode()

            l_HTML4 = etree.HTML(response)

            li_link = l_HTML4.xpath("//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/ul/li/a/@href")
            li_name = l_HTML4.xpath("//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/ul/li/a/text()")

            if li_name != []:
                for i4 in range(len(li_link)):
                    print(li_link[i4], li_name[i4])
                    print('#4' * 20)
                    list_i = list_i + '->' + li_name[i4]
                    self.level_five(li_link[i4], list_i, ioo, li_name[i4])
                    list_i = list_i.replace(f"->{li_name[i4]}", "")
                    print(55, list_i)
            else:
                print(4, list_i)
                # ioo.write(f'{list_i}\n')
                # sql = 'insert into amazon_class(`list_i`,`newtime`) values("%s", "%s")'
                # self.cursor.execute(sql, (list_i, self.newtime))
                # self.conn.commit()
                print(444)
                list_all.append(list_i)



        except Exception as g:
            self.level_four(url, list_i, ioo, name)

    def level_five(self, url, list_i, ioo, name):
        try:
            response = requests.get(url, headers=self.headers, verify=False, timeout=10).content.decode()

            l_HTML5 = etree.HTML(response)

            li_link = l_HTML5.xpath(
                "//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/ul/ul/li/a/@href")
            li_name = l_HTML5.xpath(
                "//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/ul/ul/li/a/text()")

            if li_name != []:
                for i5 in range(len(li_link)):
                    print(li_link[i5], li_name[i5])
                    print('#5' * 20)
                    list_i = list_i + '->' + li_name[i5]
                    self.level_six(li_link[i5], list_i, ioo, li_name[i5])
                    list_i = list_i.replace(f"->{li_name[i5]}", "")
                    print('66', list_i)
            else:
                print(5, list_i)
                # ioo.write(f'{list_i}\n')
                # sql = 'insert into amazon_class(`list_i`,`newtime`) values("%s", "%s")'
                # self.cursor.execute(sql, (list_i, self.newtime))
                # self.conn.commit()
                # self.ioo.close()
                print(555)
                list_all.append(list_i)

        #
        except Exception as g:
            self.level_five(url, list_i, ioo, name)

    def level_six(self, url, list_i, ioo, name):
        try:
            response = requests.get(url, headers=self.headers, verify=False, timeout=10).content.decode()

            l_HTML6 = etree.HTML(response)

            li_link = l_HTML6.xpath(
                "//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/ul/ul/ul/li/a/@href")
            li_name = l_HTML6.xpath(
                "//div[@id='zg-left-col']/ul[@id='zg_browseRoot']/ul/ul/ul/ul/ul/ul/li/a/text()")

            if li_name != []:
                for i6 in range(len(li_link)):
                    print(li_link[i6], li_name[i6])
                    print('#6' * 20)
                    list_i = list_i + '->' + li_name[i6]
            else:
                print('6', list_i)
                # ioo.write(f'{list_i}\n')
                # sql = 'insert into amazon_class(`list_i`,`newtime`) values("%s", "%s")'
                # self.cursor.execute(sql, (list_i, self.newtime))
                # self.conn.commit()
                # self.ioo.close()
                print(666)
                list_all.append(list_i)

        except Exception as g:
            self.level_six(url, list_i, ioo, name)


if __name__ == '__main__':
    obtain = Obtain()

    url = 'https://www.amazon.com/Best-Sellers/zgbs/ref=zg_bs_unv_mas_0_9408771011_2'
    obtain.level_one(url)
    # obtain.cursor.close()
    # obtain.conn.close()
    obtain.conn = pymysql.connect(host='192.168.2.205', port=3306, user='jinhuitao',
                                  password='jinhuitao_123456', database='sunway_db', charset='utf8')
    # 创建游标
    obtain.cursor = obtain.conn.cursor()
    for i in list_all:
        sql = 'insert into amazon_class(`list_i`,`newtime`) values(%s, %s)'
        print('{}插入到数据库中'.format(i))
        obtain.cursor.execute(sql, (i, obtain.newtime))
        obtain.conn.commit()
        print('{}提交数据到数据库中'.format(i))
    obtain.cursor.close()
    obtain.conn.close()
